/*
 * Copyright (C) 2011 JiangHongTiao <jjurco.sk_gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package sk.lieskove.jianghongtiao.websearch.document.preprocessing.actions;

import org.apache.log4j.Logger;
import org.htmlcleaner.*;
import sk.lieskove.jianghongtiao.common.utils.FileUtils;
import sk.lieskove.jianghongtiao.common.utils.PropertiesUtils;
import sk.lieskove.jianghongtiao.websearch.service.ExcludeReasonEnum;
import sk.lieskove.jianghongtiao.websearch.persistence.WebSearchResponse;

import java.io.File;
import java.io.IOException;

/**
 * Date of create: Oct 11, 2011
 *
 * @author JiangHongTiao <jjurco.sk_gmail.com>
 * @version 2011.1011
 */
public class HtmlClean implements PreprocessAction{
    
    private Logger log = Logger.getLogger(HtmlClean.class);
    private PropertiesUtils pu = new PropertiesUtils(HtmlClean.class);
    private long maxSize = Long.parseLong(pu.getProperty("maximal-sgml-file-size"));

    @Override
    public WebSearchResponse preprocess(WebSearchResponse response) {
        if(response.getFile() == null){
            return response;
        }
        try {
            if(response.getFile().length() > maxSize){
                response.setExcludeReasonEnum(ExcludeReasonEnum.TOO_LARGE_FILE);
                return response;
            }
            // create an instance of HtmlCleaner
            HtmlCleaner cleaner = new HtmlCleaner();
            // take default cleaner properties
            CleanerProperties props = cleaner.getProperties();
            //If this parameter is set to true, ampersand sign (&) that proceeds valid
            //XML character sequences (&XXX;) will not be escaped with &amp;XXX;
            props.setAdvancedXmlEscape(true);
            //If true, HtmlCleaner will treat SCRIPT and STYLE tag contents as CDATA
            //sections, or otherwise it will be regarded as ordinary text (special
            //characters will be escaped).
            props.setUseCdataForScriptAndStyle(true);
            //If true, special HTML entities (i.e. &ocirc;, &permil;, &times;) are
            //replaced with unicode characters they represent (ô, ‰, ×).
            //This doesn't include &, <, >, ", '.
            props.setTranslateSpecialEntities(true);
            //If true, HTML characters represented by their codes in form &#XXXX;
            //are replaced with real unicode characters (i.e. &#1078; is replaced with ж)
            props.setRecognizeUnicodeChars(true);
            //Tells whether to skip (ignore) unknown tags during cleanup.
            props.setOmitUnknownTags(false);
            //Tells whether to treat unknown tags as ordinary content, i.e. <something...>
            //will be transformed to &lt;something...&gt;. This attribute is applicable
            //only if omitUnknownTags is set to false.
            props.setTreatUnknownTagsAsContent(false);
            //Tells whether to skip (ignore) deprecated HTML tags during cleanup.
            props.setOmitDeprecatedTags(false);
            //Tells whether to treat deprecated tags as ordinary content, i.e. <font...>
            //will be transformed to &lt;font...&gt;. This attribute is applicable only
            //if omitDeprecatedTags is set to false.
            props.setTreatDeprecatedTagsAsContent(false);
            //Tells whether to skip HTML comments.
            props.setOmitComments(true);
            //Tells whether or not to put XML declaration line at the beginning of
            //the resulting XML.
            props.setOmitXmlDeclaration(true);
            //Tells whether to skip HTML declaration found in the source document.
            //If HTML document being cleaned doesn't contain one it wouldn't be placed
            //in the result anyway.
            props.setOmitDoctypeDeclaration(true);
            //Tells whether to remove HTML and BODY tags from the resulting XML, and
            //use first tag in the BODY section instead. If BODY section doesn't contain
            //any tags, then this attribute has no effect.
            props.setOmitHtmlEnvelope(false);
            //Specifies how to serialize tags with empty body - if true, compact
            //notation is used(<xxx/>), otherwise - <xxx></xxx>
            props.setUseEmptyElementTags(false);
            //Tells parser whether to allow attribute values consisting of multiple
            //words or not. If true, attribute att="a b c" will stay like it is, and
            //if false parser will split this into att="a" b="b" c="c" (this is default
            //browsers' behaviour).
            props.setAllowMultiWordAttributes(true);
            //Tells parser whether to allow html tags inside attribute values. For example,
            //when this flag is set att="here is <a href='xxxx'>link</a>" will stay like it
            //is, and if not, parser will end attribute value after "here is ".
            //This flag makes sense only if allowMultiWordAttributes is set as well.
            props.setAllowHtmlInsideAttributes(true);
            //Tells parser whether to completely ignore tags that have form <?TAGNAME....>
            //or <!TAGNAME....>. This way some HTML/XML processing instructions may be
            //omitted from the resulting xml.
            props.setIgnoreQuestAndExclam(false);
            //If true, namespace prefixes found during parsing will be preserved and
            //all neccessery xml namespace declarations will be added in the root element.
            //If false, all namespace prefixes and all xmlns namespace declarations
            //will be stripped.
            props.setNamespacesAware(true);
            //XML doesn't allow double hyphen sequence (--) inside comments. This parameter
            //tells which replacement to use for it when double hyphen is encountered
            //during parsing.
            props.setHyphenReplacementInComment("=");
            //Comma-separated list of tags that will be complitely removed (with all
            //nested elements) from XML tree after parsing. For exampe if pruneTags
            //is "script,style", resulting XML will not contain scripts and styles.
            props.setPruneTags("script,style,noscript,link");
            //Tells cleaner what value to give to boolean attributes, like checked,
            //selected and similar. Allowed values are self - value of attribute is
            //the same as attribute name (checked = "checked"), empty - attribute
            //value is empty string (checked = "") and true - value of attribute is
            //"true" (checked = "true").
            props.setBooleanAttributeValues("self");
            File outFile = FileUtils.getTempFile("HtmlClean_", "");
            TagNode clean = cleaner.clean(response.getFile());
            
            // serialize a node to a file, output stream, DOM, JDom...
            XmlSerializer serializer = new PrettyXmlSerializer(props);
            serializer.writeToFile(clean, outFile.getAbsolutePath(), "UTF-8");

            response.setFile(outFile);
        } catch (IOException ex) {
            log.error("I/O exception when trying to clean HTML/XML code");
        }
        return response;
    }
    
}
